# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import time
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
from scrapy07.request import SeleniumRequest
from selenium.webdriver import Chrome
from scrapy.http.response.html import HtmlResponse

class BossSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class BossDownloaderMiddleware:

    @classmethod
    def from_crawler(cls, crawler):
        """对中间件进行总体设置"""
        s = cls()

        # 在爬虫开始的时候. 执行spider_opened
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)

        # 在爬虫结束的时候. 执行spider_closed
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)

        return s

    def process_request(self, request, spider):
        """处理请求"""
        if isinstance(request, SeleniumRequest):

            # selenium发送请求
            self.web.get(request.url)
            time.sleep(3)

            # 获取页面源码
            page_source = self.web.page_source

            # 包装成 Response，往引擎中传
            return HtmlResponse(url=request.url, # 响应的地址
                                encoding='utf-8', # 编码格式
                                request=request, # 回应的请求
                                body=page_source) # 返回的源码

    def process_response(self, request, response, spider):
        """处理响应"""
        return response

    def process_exception(self, request, exception, spider):
        """处理异常"""
        pass

    def spider_opened(self, spider):
        """爬虫开启时，执行该函数"""

        # 创建浏览器
        self.web = Chrome()

        # 隐式等待
        self.web.implicitly_wait(10)

        print("创建浏览器")

    def spider_closed(self, spider):
        """爬虫结束时，执行该函数"""
        self.web.close()
        print("关闭浏览器")
